"""
  Author:     Adrian Mandy <jatmandy@gmail.com>
  Program:    Library of functions to manipulate documents images and summaries
              for litigation software packages
  Date:       January 15, 2006

  Description:  This library contains classes for manipulating lists of images
                and summaries and generating load files in various formats for
                litigation software packages like Summation.

                
                todo:
                - create table on load
                - add fields or ignore fields that don't exist in table
                - fix error on importing long text lines
"""

import os
import sys
import re
import types
import math
from string import Template
from PyQt4 import QtSql, QtGui, QtCore

class dbSql:
    """Class for interfacing with database."""
    def __init__(self, dbFile="test.db"):
        """Constructor for dbSql
        """
        #self.tableName = "Summaries"
        self.createConnection(dbFile)
        print( "Open: ", self.db.open())
        self.query = QtSql.QSqlQuery(self.db)
        self.table = QtSql.QSqlTableModel()
        self.table.EditStrategy(QtSql.QSqlTableModel.OnRowChange)
        self.fieldre = re.compile("(?:\(|,)(.*?)(?=,|\))")
        
    def __del__(self):
        self.db.close()
        
    def setTable(self, table,  keyTag="Begdoc"):
        self.keyTag = keyTag
        self.tableName = table
        self.table.setTable(table)
        self.table.select()

    def getTable(self,  tableName):
        table = QtSql.QSqlTableModel(None, self.db)
        table.EditStrategy(QtSql.QSqlTableModel.OnRowChange)
        table.setTable(tableName)
        table.select()
        return table
        
    def createConnection(self, fileName):
        self.db = QtSql.QSqlDatabase.addDatabase("QSQLITE")
        #self.db.setDatabaseName(":memory:")
        self.db.setDatabaseName(fileName)
        if not self.db.open():
            QtGui.QMessageBox.critical(None, QtGui.qApp.tr("Cannot open database"),
                    QtGui.qApp.tr("Unable to establish a database connection.\n"
                                  "This software needs SQLite support. Please read "
                                  "the Qt SQL driver documentation for information "
                                  "how to build it.\n\nClick Cancel to exit."),
                    QtGui.QMessageBox.Cancel, QtGui.QMessageBox.NoButton)
            return False
        return True

    def buildValues(self,  record):
        return ''.join(["'"+record[field].replace("'", "''")+"'," for field in record.keys()])[:-1]


#        self.query.exec_("select sql from sqlite_master where name = '"+table+"'")
#        self.query.first()
#        fieldNames = self.fieldre.findall(str(self.query.value(0).toString()))
    def Prepare(self, table,  fieldNames):
        qry = "INSERT INTO "+table+" ("+','.join(fieldNames)+") VALUES (:"+',:'.join(fieldNames)+")"
        self.query.prepare(qry)

    def Add(self, record):
        """Add a new record to database
        
            >>> Add("Doc-44",{"Begdoc#":"Doc-44","Doctitle":"Test document"})
        """
        for key, value in record:
            self.query.bindValue(":"+key, value);
        self.query.exec_()

    def keys(self,  keyTag,  tableName):
        """Return a list of keys
        
            >>> keys()
            ['Doc-16', 'Doc-17', 'Doc-14', 'Doc-15', 'Doc-12', 'Doc-13', 'Doc-10', 'Doc-11']
        """
        result = []
        self.query.exec_("select "+keyTag+" from "+tableName)
        while self.query.next():
            result.append(str(self.query.value(0).toString()))
        return result

    def has_key(self, key):
        """Check if key exists
        
            >>> has_key('Doc-14')
            True
        """
        query = QtSql.QSqlQuery(self.db)
        query.exec_("select "+self.keyTag+" from "+ self.tableName +" where "+self.keyTag+" = '" + key + "'")
        has_k = query.first()
        query.clear()
        return has_k

    def has_table(self,  tableName):
        """Check if table exists
        
            >>> has_key('Summaries')
            True
        """
        return tableName in self.db.tables(QtSql.QSql.Tables)

    def create_table(self,  tableName,  fields):
        if not self.has_table(tableName):
            qry = "create table "+tableName+"("+''.join([field+"," for field in fields])[:-1]+");"
            return self.query.exec_(qry)
#            if self.query.exec_(qry):
#                print "Create Success"
#            else:
#                print "Create Failed"
        else:
            for field in fields:
                if not self.has_field(field,  tableName):
                    self.create_field(tableName,  field)

    def fields(self,  tableName):
        """Generate list of field names
            
            >>> fields()
            ['Begdoc#', 'Doctitle']
        """
        table = QtSql.QSqlTableModel()
        table.setTable(tableName)
        table.select()
        rec = table.record(0)
        fields = []
        for fieldIndex in range(rec.count()):
            fields.append(str(rec.fieldName(fieldIndex)))
        return fields

    def has_field(self, testField,  tableName):
        return self.db.record(tableName).contains(testField)

    def create_field(self,  tableName,  field):
        qry = "alter table "+tableName+" add "+field+";"
        return self.query.exec_(qry)
#        if self.query.exec_(qry):
#            print "Alter Success"
#        else:
#            print "Alter Fail"

    def fieldSize(self, field):
        """Return largest size of field

            >>> fieldSize('Doctitle')
            124
        """
        self.query.exec_("select MAX(len gth("+ field +")) as MaxLength from "+ self.tableName)
        self.query.next()
        return self.query.value(0).toInt()[0]

    def record(self, key,  tableName, keyTag):
        """Return a record
        
            >>> record("Doc-13")
            {'Begdoc#': 'Doc-13', 'Doctitle': "IBM's Answer April 30, 2003"}
        """
        fields = self.fields(tableName)
        qry = "select " + ''.join([str(field) +', ' for field in fields])[:-2] + " from "+ tableName + " where " + keyTag + " = '" + key + "'"
        self.query.exec_("select " + ''.join([str(field) +', ' for field in fields])[:-2] + " from "+ tableName + " where " + keyTag + " = '" + key + "'")
        self.query.next()
        rec = self.query.record()
        values = [str(rec.value(field).toString().replace('\xa0',' ')) for field in fields]
        result = dict(zip(fields, values))
        return result

    def update(self, key, field, value):
        """Update a field in a record
        
            >>> update("Doc-13", 'Doctitle', "IBM's Answer April 1, 2005")
        """
        #print self.tableName, field, value, self.keyTag, key
        return self.query.exec_("update "+ self.tableName +" set "+ field +" = '"+ value +"' where "+ self.keyTag +" = '"+ key +"'")

    def field(self, tablename,  key, field,  keyTag):
        """Return a field from a record
        
            >>> field("Doc-13", 'Doctitle')
            "IBM's Answer April 1, 2005"
        """
        return self.record(key,  tablename,  keyTag)[field]

    def delete(self,  tableName, keys=[]):
        print(tableName)
        return self.query.exec_("delete from "+ tableName + ";")

    def drop(self,  tableName):
        return self.query.exec_("DROP TABLE "+tableName)

    def prefix(self, tableName,  fieldName,  prefix):
        return self.query.exec_("UPDATE "+tableName+" SET "+fieldName+"='"+prefix+"' || "+fieldName)
        
    def begin(self):
        self.db.transaction()
        
    def commit(self):
        self.db.commit()
        
    def rollback(self):
        self.db.rollback()
        
class ImageNames:
    """Class for formatting and manipulating filenames."""
    def __init__(self, ImageFiles, defdir):
        """Constructor for ImageNames.
        
        ImageFiles -- list of image file names, must not have a path component
            eg. 'A001.tif;B001.tif;C00{1-9}.tif' not '0034\\A0012.tif'
        defdir -- the case path where all the images are stored
        """
        self.__ImageFiles =[]
        self.__defdir = defdir.replace("\\", os.path.sep)
        self.__prefix = 0
        self.__page = 1
        self.Add(ImageFiles)

    def __rangeExpand(self, diiRange):
        """Produce a list of individual filenames given a range
        """
        rPat = re.compile('(.*?)\\{([0-9]+)-([0-9]+)\\}(.*)')
        rParts = rPat.search(diiRange)
        if rParts:
            fparts = rParts.group(1, 2, 3, 4)
            return [fparts[0]+str(n).zfill(len(fparts[1]))+fparts[3] for n in range(int(fparts[1]), int(fparts[2])+1)]
        else:
            return [diiRange]

    def __fullList(self, diiRanges):
        """convert list of file ranges to list of files
        """
        #print "New:", [diiList(name, extension) for name in diiRanges]
        result = []
        for name in diiRanges:
            result = result + self.__rangeExpand(name)
        return result

    def __fixPath(self, fileList, custPath, defdir):
        """Build the full path of a list of files with default directory and the custom path
        """
        return [os.path.normpath(os.path.join(defdir, name).replace("@I", custPath)) for name in fileList]

    def __pagesRange(self, pageRange):
        """Take a start and end page and build range.

        eg (('-','001'),('-','007'))
        -> -00{1-7}
        """
        p1 = pageRange[0][self.__page]
        p2 = pageRange[1][self.__page]
        #build document range string if more than one page
        if p1 == p2:
            return pageRange[0][self.__prefix] + pageRange[0][self.__page]
        elif len(p1) != len(p2):
            return pageRange[0][self.__prefix] + "{" + p1 + "-" + p2 + "}"
        else:
            preflag = True
            prelen = 0
            for i, j in zip(p1, p2):
                if (i==j) and preflag:
                    prelen += 1
                else:
                    preflag = False
            rngprefix = p1[:prelen]
            #rngprefix = ''.join([i for i, j in zip(p1, p2) if i==j])
            pstart = p1[prelen:]
            #pstart =    ''.join([i for i, j in zip(p1, p2) if i!=j])
            pend = p2[prelen:]
            #pend =      ''.join([j for i, j in zip(p1, p2) if i!=j])
            #remove leading zeros
            while pstart[0] == '0':
                pstart = rngprefix[-1] + pstart
                pend = rngprefix[-1] + pend
                rngprefix = rngprefix[:-1]
            return pageRange[0][self.__prefix] + rngprefix + "{" + pstart + "-" + pend + "}"

    def __pageDiff(self, i, j):
        """Find the numerical difference between two strings of digits
        """
        if i.isdigit() and j.isdigit():
            return int(i) - int(j)
        else:
            return 0

    def __power10(self, i):
        """Return true if string of digits is a power of 10
        """
        if i.isdigit():
            if int(i) == 0:
                return True
            else:
                return math.modf(math.log10(int(i)))[0] == 0.0
        else:
            return False

    def __gindex(self, list):
        """Find the index of the group with a difference
        """
        blist = [i!=0 for i in list]
        if blist.count(True) == 1:
            return blist.index(True)
        else:
            return -1

    def Add(self, images):
        """Add image files from semicolon separated list.

        images -- semicolon separated list of files eg. "A001.tif;A002.tif;A003.tif"
        """
        if type(images) is str:
            images = images.split(';')
        for j in images:
            self.__ImageFiles.append(os.path.normcase(j))

    def ExistingFileNames(self, ImagePath):
        """Enumerate the list of files and return those that exist in the filesystem.
        """
        return [filename for filename in self.FullFileNames(ImagePath) if os.path.isfile(filename)]

    def Extensions(self):
        """Return a list of all the different extensions
        """
        return reduce(lambda l, x: x not in l and l.append(x) or l, [os.path.splitext(fname)[1] for fname in self.FileNames()], [])

    def FileNames(self):
        """Return a list of all filenames with no path.
        """
        result = []
        for i in self.__ImageFiles:
            for j in i.split(';'):
                if j.rfind("\\") > -1:
                    j = j.rsplit("\\",1)[1]
                result.append(j)
        return self.__fullList(result)

    def FullFileNames(self, ImagePath):
        """Return the full path and filename to each image file.

        This function will add the defdir to the filename and replace the @I
        with the ImagePath name
        ImagePath -- the base path to all the images
        """
        return self.__fixPath(self.FileNames(), ImagePath, self.__defdir)

    def Homogenous(self):
        """Are the file extensions the same for all the image files?
        """
        return len(self.Extensions()) == 1

    def ImageFileList(self):
        """Generate semicolon separated list of image filenames.
        """
        def delim(x,y): return x + ';' + y
        return reduce(delim, self.PageRanges())

    def MissingFileNames(self, ImagePath):
        """Enumerate the list of files and return those that are missing from the filesystem.
        """
        return [filename for filename in self.FullFileNames(ImagePath) if not os.path.isfile(filename)]

    def MissingFiles(self, ImagePath):
        """Are any of the images missing actual files from the filesystem.
        """
        return (len(self.MissingFileNames(ImagePath)) > 0)

    def PageCount(self):
        """Count the number of pages by counting the total image filenames.
        """
        return len(self.FileNames())

    def PageRanges(self):
        """Parse a list of filenames into ranges for use in diis.

        eg 003_001.tif;003_002.tif;003_003.tif
        -> 003_00{1-3}.tif
        """
        #split filenames into components
        x1 = [[j for j in re.findall("([^0-9]*)([0-9]*)", i) if len(reduce(lambda x, y: x + y, j)) > 0] for i in self.FileNames()]
        #find the different prefix boundaries
        diffPrefix = [reduce(lambda x, y: x or y, [(k[0] != l[0]) for k, l in zip(i, j)]) for i, j in zip(x1[1:], x1[:-1])]
        #Calculate how much each pair of groups differ for a pair of names
        n = [[int(k[1]) - int(l[1]) for k, l in zip(i, j) if (len(k[1]) > 0) and (len(l[1]) > 0)] for i, j in zip(x1[1:], x1[:-1])]
        #Calculate total diff for all groups in one pair of names
        pageDiff = [reduce(lambda x, y: abs(x) + abs(y), i) != 1 for i in n]
        #Combine boundaries
        diiFull = [reduce(lambda x, y: x or y, l) for l in zip(diffPrefix, pageDiff)]
        #Differences move to different group?
        groupDD = map(self.__gindex, n)
        groupDD = [1] + groupDD
        groupsDiffer = [False] + [(i != j) and not k for i, j, k in zip(map(self.__gindex, n)[1:], map(self.__gindex, n)[:-1], diiFull)] + [True]
        #Find powers of 10 boundaries
        pow10 = [self.__power10(y[x][1]) for x, y in zip(groupDD, x1)][1:] +[False]
        #find the boundaries where number of groups changes
        groupMatch = [i[0] != i[1] for i in zip(map(len, x1)[1:], map(len, x1)[:-1])]
        #combine all conditions
        diffFull = [reduce(lambda x, y: x or y, n) for n in zip(groupMatch, groupsDiffer, pageDiff, diffPrefix, pow10)]
        groupRanges = zip([i for i, p in enumerate([True] + diffFull) if p], [i for i, p in enumerate(diffFull + [True]) if p]) #only compare groups with same sizes
        return [reduce(lambda x, y: x + y, [self.__pagesRange((s, e)) for s, e in zip(x1[x], x1[y])]) for x, y in groupRanges]

    def RealPageCount(self, ImagePath):
        """Count the number of pages excluding those with missing files in the file system.
        """
        return len(self.ExistingFileNames(ImagePath))

class Summaries:
    """Class for reading summaries."""
    def __init__(self, db=None, tableName="Summaries", keyTag="Imgtag", remQuotes=True, delims=[',','"',';','~']):
        """Constructor for summaries

        filename -- text file to read summaries from
        keytag -- file to use when linking images
        remQuotes -- flag should quotes be removed from fields?
        delims -- delimeters used in text load file
        """
        self.tableName = tableName
        self.removeQuotes = remQuotes
        self.keyTag = keyTag
        self.Lines = db

    def table(self):
        """Returns a table model
        """
        return self.Lines.getTable(self.tableName)

    def formatline(self, fields, delims=[',','"',';','~']):
        """Format fields in preparation for output to a text file

        fields -- list of fields to be formatted
        delims=[field delimiter, quote character, multi-entry delimiter, return value in data]
        """
        #double the quote character to allow decoding later
        fields = [f.replace(delims[1], delims[1]+delims[1]) for f in fields]
        #replace line breaks with linebreak delimiter
        fields = [f.replace('\n', delims[3]) for f in fields]
        #surround fields with quote characters
        def delim(x,y): return x + delims[1] + delims[0] + delims[1] + y
        return delims[1]+reduce(delim, fields)+delims[1]

    def Add(self, record):
        """Add a new summary

        key -- add new summary using this key
        record -- contains summary
        """
        fields = [i[0] for i in record]
        self.Lines.Prepare(self.tableName, fields)
        self.Lines.Add(record)

    def Read(self, fileName, delims=[',','"',';','~'], fregex="(${q}(?:[^${q}]|${q}${q})*${q}|(?<=${f})[^${f}]*(?=$$)|[^${f}]*(?=${f}))(?:${f}|$$)?"):
        """Read a text file containing summaries
        """
        retemplate = Template(fregex)
        refieldstring= retemplate.substitute(f=delims[0], q=delims[1])
        resplit = re.compile(refieldstring)
        retemplate = Template("${q}(.*)${q}")
        delimStrip = retemplate.substitute(q=delims[1])
        redelimStrip = re.compile(delimStrip)
        
        f = open(fileName, 'r')
        fields = f.readline()
        fields = resplit.findall(fields.strip())
        print(fields)
        self.Lines.create_table(self.tableName,  fields)
        print("Begin:",self.Lines.begin())
        self.Lines.Prepare(self.tableName,  fields)
        for line in f:
            print(line.strip())
            values = resplit.findall(line.strip())
            print(values)
            values = [redelimStrip.sub('\\1', v) for v in values]
            values = [v.replace(delims[1]+delims[1],delims[1]) for v in values]
            values = [v.replace(delims[3],'\n') for v in values]
            record = zip(fields, values)
            print(record)
            #try:
            self.Lines.Add(record)
            serror = str(self.Lines.db.lastError().text()).strip()
            if len(serror)>0:
                print(serror)
            
            #except:
                #print "Error:", record
                #self.Lines.rollback()
                #continue
        print("Commit:",self.Lines.commit())
        print(self.Lines.db.lastError().text())

    def purge(self):
        self.Lines.delete(self.tableName)

    def keys(self):
        """Return a list of keys
        """
        return self.Lines.keys(self.keyTag,  self.tableName)

    def has_key(self, key):
        """Check if key exists
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        return self.Lines.has_key(key)

    #change to join?
    def filter(self, filterList, filterType="i"):
        """Summaries using another list of summarie's keys

        filterType -- 'i' intersection join (filter contains keys to be produced)
                'd' difference join (filter contains keys to be removed)
                'u' union join (filter list is combined with summaries)
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        flist = Summaries()
        if filterType == "i":
            for key in filterList.keys():
                flist.Add(key, self.Lines.record(key))
        else:
            for key in self.keys():
                if not filterList.has_key(key):
                    flist.Add(key, self.Lines.record(key))
        return flist

    def fields(self):
        """Generate list of field names
        """
        return self.Lines.fields(self.tableName)

    def fieldSize(self, field):
        """Return largest size of field
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        return self.Lines.fieldSize(field)

    def out2(self, model,  f=sys.stdout, delims=[",",'"',";","~"], quotenames=False):
        """Output summaries to a text file

        f -- file handle to write to
        delims=[field delimiter, quote character, multi-entry delimiter, return value in data]
        quotenames -- should field names be quoted?
        """
        fieldlist = self.fields()
        #print fieldlist
        if quotenames:
            f.write(self.formatline(fieldlist, delims)+"\n")
        else:
            f.write(self.formatline(fieldlist, [delims[0], '', delims[2], delims[3]])+"\n")

        #print self.keys()
        for key in self.keys():
            f.write(self.formatline([self.Lines.field(self.tableName,  key, fkey,  self.keyTag) for fkey in fieldlist], delims)+"\n")

    def out(self, activeModel, f=sys.stdout, delims=[",",'"',";","~"], quotenames=False):
        while activeModel.canFetchMore():
            activeModel.fetchMore()

        columnCount = activeModel.columnCount()
        fieldlist = [str(activeModel.headerData(cindex, QtCore.Qt.Horizontal).toString()) for cindex in range(columnCount)]

        #print fieldlist
        if quotenames:
            f.write(self.formatline(fieldlist, delims)+"\n")
        else:
            f.write(self.formatline(fieldlist, [delims[0], '', delims[2], delims[3]])+"\n")
        rowe = activeModel.rowCount()
        for row in range(0, rowe):
            rowdata = [str(activeModel.data(activeModel.createIndex(row, column)).toString()) for column in range(columnCount)]
            f.write(self.formatline(rowdata, delims)+"\n")
        
    def diifields(self, f=sys.stdout, key=None, special=dict([])):
        """Output simple dii file.

        special -- Special handling {"Imgtag":"@T ", "Defdir":"@D ", "Imgfiles":None}
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        record = self.Lines.record(key,  self.tableName,  self.keyTag).copy()
        for sfield in special.keys():
            if record.has_key(sfield):
                fpop = record.pop(sfield)
                if not (special[sfield] == None):
                    f.write(special[sfield] + fpop +"\n")
        for field in record.keys():
            if len(record[field])>0:
                f.write("@C " + field + " " + record[field] +"\n")

class Images(Summaries):
    """Class for working with images."""
    def __init__(self, db=None, tableName="Images", imagePath=None, keyTag="Imgtag", remQuotes=True, regex='(([a-zA-Z]+-[0-9]{2})[0-9]+.*)', delims=[',','"',';','~']):
        """Constructor for images list

        filename -- text file with image list to read
        imagePath -- base path for images
        keyTag -- field used to link images with summaries
        regex -- regular expression that splits filename into (whole filename, key) eg. ('0123-01.tif', '0123')
        '(([0-9]+[a-zA-Z]*_[0-9]{3}[a-zA-Z]*)_[0-9]+.*)'
        """
        Summaries.__init__(self, db, tableName, keyTag, remQuotes, delims)
        self.imagePath = imagePath
        self.diiList = dict([])
        self.reg = re.compile(regex)
        self.file = 0
        self.key = 1
        self.relativePath = "@I"

    def ReadImages(self, fileName, delims=[',','"',';','~']):
        """Read a delimited text file.
        """
        Summaries.Read(self, fileName, delims)
        self.Lines.begin()
        for key in self.keys():
            #self.Lines[key]['Imgfiles'] = ImageNames(self.Lines[key]['Imgfiles'], self.Lines[key]['Defdir'])
            self.Lines.update(key, 'Imgfiles', self.Lines.field(key, 'Imgfiles'))
        self.Lines.commit()

    def RealFiles(self, key):
        """Return a list of files that exist in the filesystem.
        """
        imgs = imageNames(self.Lines.field(key, 'Imgfiles'))
        return imgs.ExistingFileNames(self.imagePath)

    def ImaginaryFiles(self, key):
        """Return a list of files that don't exist in the filesystem.
        """
        imgs = imageNames(self.Lines.field(key, 'Imgfiles'))
        return imgs.MissingFileNames(self.imagePath)

    def PageCount(self, key):
        """What are the number of pages for a given summary.
        """
        imgs = imageNames(self.Lines.field(key, 'Imgfiles'))
        return imgs.PageCount()

    def Homogenous(self, key):
        """Are all of the images for a given key of the same image type.
        """
        if self.has_key(key):
            imgs = imageNames(self.Lines.field(key, 'Imgfiles'))
            return imgs.Homogenous()
        else:
            return False

    def filter(self, filterList, filterType='i'):
        """Filter image object and return new image object.
        """
        flist = Images()
        if filterType == "i":
            for key in filterList.keys():
                flist.Add(key, self.Lines.record(key))
        else:
            for key in self.keys():
                if not filterList.has_key(key):
                    flist.Add(key, self.Lines.record(key))
        return flist

    def List(self, f=sys.stdout):
        """Output a list of images so they can be copied to a new location
        """
        for key in self.Lines.keys(self.keyTag, self.tableName):
            imgs = ImageNames(self.Lines.field(self.tableName, key, 'Imgfiles', self.keyTag), "dir/")
            npath = self.Lines.field(self.tableName, key, 'Defdir', self.keyTag)
            pageFilenames = imgs.FileNames()
            for npage in pageFilenames:
                f.write("@command "+os.path.join(npath, npage)+os.path.join(" @dest", npage) + "\n")

    def imagesOut(self, f=sys.stdout, delims=[",",'"',";","~"], quotenames=False):
        """Output image table.
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        fieldlist = [fld for fld in self.fields()]
        if quotenames:
            f.write(self.formatline(fieldlist, delims)+"\n")
        else:
            f.write(self.formatline(fieldlist, [delims[0], '', delims[2], delims[3]])+"\n")
        for key in self.keys():
            record = []
            for fkey in fieldlist:
                if fkey == 'Imgfiles':
                    print(self.Lines.field(self.tableName,  key, fkey,  self.keyTag))
                    imgs = ImageNames(self.Lines.field(self.tableName,  key, fkey,  self.keyTag), "dir/")
                    print(imgs.FileNames())
                    print(imgs.PageRanges())
                    record.append(imgs.ImageFileList())
                    #record.append(self.Lines.field(key, fkey))
                else:
                    record.append(self.Lines.field(self.tableName,  key, fkey,  self.keyTag))
            f.write(self.formatline(record, delims)+"\n")

    def diifiles(self, f=sys.stdout, key=None):
        """Output simple dii file record.
        """
        self.Lines.setTable(self.tableName,  self.keyTag)
        imgs = ImageNames(self.Lines.field(self.tableName, key, 'Imgfiles',  self.keyTag), "defdir/")
        for file in imgs.PageRanges():
            f.write(file + '\n')

    def fparse(self, files):
        """Parse a list of files into component parts.
        """
        result = dict([])
        # bad file specs that don't match the regex
        self.fbad = [f for f in files if not self.reg.search(f)]
        # split good filenames into parts (whole filename, key)
        filparts = [self.reg.search(f).groups() for f in files if self.reg.search(f)]
        while len(filparts) > 0:
            currentTag = filparts[0][self.key]
            result[currentTag] = [i[self.file] for i in filparts if i[self.key] == currentTag]
            filparts = [i for i in filparts if i[self.key] != currentTag]
        return result

    def fileSort(self, v):
        return v.split('.')[0].zfill(8)

    def dirWalk(self, imageDir, dir, filetype=[".tif",".jpg",".gif","bmp"], padded="", pathkey=False):
        """Walk a directory structure and find images add them to list
        """
        imageDir = os.path.normcase(imageDir)
        dir = os.path.normcase(dir)
        self.Lines.begin()
        self.Lines.create_table(dir, ['Imgtag','Defdir','Imgfiles'])
        for root, dirs, files in os.walk(os.path.join(imageDir, dir)):
            files.sort(key=self.fileSort)
            files = [os.path.normcase(f) for f in files]
            files = [f for f in files if os.path.splitext(f)[1] in filetype]
            if len(files) > 0:
                result = self.fparse(files)
                print(result)
                #key found in path?
                defdir = root
                if root.find(imageDir) ==0:
                    defdir = root.replace(os.path.normpath(imageDir),  self.relativePath,  1)
                if pathkey:
                    ky = os.path.basename(root)
                    for k in result.keys():
                        imgs = ImageNames(result[k],imageDir)
                        self.Add([('Imgtag',padded + ky),('Defdir',defdir), ('Imgfiles',imgs.ImageFileList())])
                else:
                    for k in result.keys():
                        imgs = ImageNames(result[k],imageDir)
                        self.Add([('Imgtag',padded + k),('Defdir',defdir), ('Imgfiles',imgs.ImageFileList())])
        self.Lines.commit()

class Case(Images):
    """Class for creating and manipulating image file lists"""
    def __init__(self, db=None, tableName=None, imagePath=None, keyTag="Imgtag", remQuotes=True, regex='(([a-zA-Z]+-[0-9]{2})[0-9]+.*)', delims=[',','"',';','~']):
        """diiFile constructor

        fileName -- dii file to read
        keyTag -- Summary field to use when linking images with summaries
            is always "Imgtag" for image list
        """
        Images.__init__(self, db, tableName, imagePath, keyTag, remQuotes, regex, delims)
        #self.db = db
        #self.Summaries = Summaries(db=self.db, keyTag=keyTag)
        #self.Images = Images(db=self.db)
        self.Fulltext = None

    def tagOut(self, outfile=sys.stdout, filter=[], special=dict([])):
        """Output a file in dii format

        d = diiFile('k:\\database\\work\\pad\\img.txt')
        d.diiout([],open('c:\\test2.txt','w'))
        """
        special.update({self.keyTag:None, "Imgtag":"@T ", "Defdir":"@D ", "Imgfiles": None})
        if len(filter) > 0:
            keys = filter
        else:
            keys = self.keys()
        keys.sort()
        count = 1
        for key in keys:
            if self.has_key(key):
                outfile.write("; Record " + str(count) +"\n")
                self.diifields(outfile, key, special)
                #if self.Summaries.has_key(key):
                #    self.Summaries.diifields(outfile, key, special)
                if (count == 1) and (self.Fulltext != None):
                    outfile.write("@FULLTEXT " + self.fulltext +"\n")
                self.diifiles(outfile, key)
                outfile.write("\n")
                count = count + 1

    def update(self, diirec):
        """write the variouse dii field into the appropriate summaries or images databases
        """
        tag = diirec['Imgtag']
        sumrec={}
        imgrec={}
        for key in diirec.keys():
            if key == 'Imgtag':
                imgrec[key] = diirec[key]
                sumrec[self.Summaries.keyTag] = diirec[key]
            elif key == 'Defdir':
                imgrec[key] = diirec[key]
            elif key == 'Imgfiles':
                imgrec[key] = reduce(lambda x, y: x + ";" + y, diirec[key])
                #imgrec[key] = diirec[key]
            elif key == 'PGCOUNT':
                imgrec['Pages'] = diirec[key]
                sumrec[key] = diirec[key]
            elif key == 'Pages':
                #nothing
                print("-Pages-")
            elif key == 'id':
                #nothing
                print("-id-")
            else:
                sumrec[key] = diirec[key]
        print(sumrec)
        self.Summaries.Add(sumrec)
        self.Images.Add(imgrec)

    def convertTags(self, wholeline):
        """convert dii tags to database field names
        """
        line = wholeline.split(" ", 2)
        if line[0] == "@C":
            return [[line[1],line[2]]]
        elif line[0] == "@T":
            if len(line) > 2:
                return [["Imgtag",line[1]],["Pgcount", line[2]]]
            else:
                return [["Imgtag",line[1]]]
        elif line[0] == "@D":
            return [["Defdir",line[1]]]
        else:
            print(wholeline)
            return [["NA",wholeline]]

    def recAddField(self, rec, line):
        keyValues = self.convertTags(line)
        for keyvalue in keyValues:
            if keyvalue[0] == '@FULLTEXT':
                self.fulltext = keyvalue[1]
            else:
                rec[keyvalue[0]] = keyvalue[1]

    def readTagFile(self, fileName):
        """Read a tag file eg. dii
        
        @FULLTEXT (DOC|PAGE)
        @C <field> <value> = <field>:<value>
        @T=Imgtag Pgcount
        @D=Defdir
        123.tif  \
        124.tif    -> Imgfiles
        125.tif  /
        
        """
        newrec = True
        tagFile = file(fileName, 'r')
        diirec = {}
        self.Lines.begin()
        for line in tagFile:
            line = line.split("\n")[0]
            if len(line) == 0 or (line[0] == "@" and newrec): #looks like we're at the start of new record so save current record
                if diirec.has_key("Imgtag"):
                    #self.update(diirec)
                    self.Lines.create_table(self.tableName, diirec.keys())
                    print(diirec["Imgfiles"], diirec["Defdir"])
                    imgs = ImageNames(diirec["Imgfiles"], diirec["Defdir"])
                    diirec["Imgfiles"] = imgs.ImageFileList()
                    self.Lines.Prepare(self.tableName,  diirec.keys())
                    self.Lines.Add(diirec.items())
                    serror = str(self.Lines.db.lastError().text()).strip()
                    if len(serror)>0:
                        print(serror)

                    print(diirec)
                else:
                    print("Bad record:", diirec)
                diirec = {}
                newrec = False
                if len(line) > 0:
                    self.recAddField(diirec, line)
            elif line[0] == "@": #add tag lines
                self.recAddField(diirec, line)
            else: #add image files
                newrec = True
                if diirec.has_key("Imgfiles"):
                    diirec["Imgfiles"] = diirec["Imgfiles"] + [line]
                else:
                    diirec["Imgfiles"] = [line]
        if diirec.has_key("Imgtag"):
            #self.update(diirec)
            self.Lines.create_table(self.tableName, diirec.keys())                    
            imgs = ImageNames(diirec["Imgfiles"], diirec["Defdir"])
            diirec["Imgfiles"] = imgs.ImageFileList()
            self.Lines.Add(diirec.items())
            print(diirec)
        self.Lines.commit()
        tagFile.close()

    def concRead(self, fileName, delims=[',','"',';','~'], fregex="(${q}(?:[^${q}]|${q}${q})*${q}|(?<=${f})[^${f}]*(?=$$)|[^${f}]*(?=${f}))(?:${f}|$$)?"):
        """Read a Concordance load file
        """
        retemplate = Template(fregex)
        refieldstring = retemplate.substitute(f=delims[0], q=delims[1])
        resplit = re.compile(refieldstring)
        retemplate = Template("${q}(.*)${q}")
        delimStrip = retemplate.substitute(q=delims[1])
        redelimStrip = re.compile(delimStrip)

        #contents= file(fileName, 'r')
        #for record in contents:
        #    values = resplit.findall(record.strip())
        #    print values


        tagFile = file(fileName, 'r')
        diirec = {}
        self.Lines.begin()
        for line in tagFile:
            values = resplit.findall(line.strip())
            tag = values[0]
            volume = values[1]
            image = values[2]
            pgcount = values[6]
            keytag = values[3]
            
            if keytag == "Y":
                if diirec.has_key("Imgtag"):
                    #save previous record
                    self.Lines.create_table(self.tableName, diirec.keys())
                    print(diirec["Imgfiles"], diirec["Defdir"])
                    imgs = ImageNames(diirec["Imgfiles"], diirec["Defdir"])
                    diirec["Imgfiles"] = imgs.ImageFileList()
                    self.Lines.Prepare(self.tableName,  diirec.keys())
                    self.Lines.Add(diirec.items())
                    serror = str(self.Lines.db.lastError().text()).strip()
                    if len(serror)>0:
                        print(serror)
    
                    print(diirec)
                else:
                    print("Bad record:", diirec)
                diirec = {}

                #new tag
                diirec["Imgtag"] = tag
                diirec["Defdir"] = os.path.join(volume, os.path.dirname(image)[1:])
            
            if diirec.has_key("Imgfiles"):
                diirec["Imgfiles"] = diirec["Imgfiles"] + [image]
            else:
                diirec["Imgfiles"] = [image]
        if diirec.has_key("Imgtag"):
            self.Lines.create_table(self.tableName, diirec.keys())                    
            imgs = ImageNames(diirec["Imgfiles"], diirec["Defdir"])
            diirec["Imgfiles"] = imgs.ImageFileList()
            self.Lines.Add(diirec.items())
            print(diirec)
        self.Lines.commit()
        tagFile.close()

            
    def idsRead(self, fileName):
        def getDelim(field):
            datefields = ['date','docdate']
            dotfields = ['textid']
            if datefields.count(field) >0:
                return '/'
            if dotfields.count(field) >0:
                return '.'
            else:
                return '\n'

        contents = file(fileName, 'r')
        summaryRecord = dict([])
        records = 0
        self.Lines.begin()
        for field in contents:
            field = field.strip()
            if field.startswith("###"):
                #end of current summary
                self.Lines.create_table(self.tableName, summaryRecord.keys())
                self.Lines.Prepare(self.tableName,  summaryRecord.keys())
                self.Lines.Add(summaryRecord.items())
                summaryRecord = dict([])
                records = records + 1
            else:
                fieldParts = idsField(field)
                if fieldParts[2]:
                    if summaryRecord.has_key(fieldParts[0]):
                        summaryRecord[fieldParts[0]] = summaryRecord[fieldParts[0]] +getDelim(fieldParts[0])+ fieldParts[2]
                    else:
                        summaryRecord[fieldParts[0]] = fieldParts[2]
        self.Lines.commit()
        contents.close()






"""Titan stuff"""        
"""
Tag File (dii, ids)
dii:
@T = key field
@C = named field
"""

def idsField(field):
    if field.find(":") > 0:
        return [field.split(":",1)[0]] + field.split(":",1)[1].split("=",1)
    elif field.find("=") > 0:
        return [field.split("=",1)[0], '0', field.split("=",1)[1]]

#titan2(open("z:\\trialex2p.ids", 'r'), 'test')
def titan2(titanids, idfield):
    titandb = {}
    temprecord = {}
    for titanline in titanids:
        if not titanline.startswith("###"):
            lineparts = titanline.strip().split('=',1)
            value = lineparts[1]
            field = lineparts[0].split(':', 1)
            if len(field)> 1:
                fieldIndex = field[1]
            else:
                fieldIndex = "0"
            field = field[0]
            print(field, fieldIndex, value)
        


#titanTranConvert(open("g:\\Database\\TitanConvert\\1-Grace\\1-Privest\\privtran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\1-Grace\\2-Bentall\\benttran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\1-Grace\\3-RoyalMuseum\\roymustran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\1-Grace\\4-Laurelton\\laurtran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\1-Grace\\5-WRGrace\\wrgtran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\4-Amswiss\\amswisstran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\5-Alcan\\alcantran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\6-Continental\\contitran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\7-Fletcher\\fletchertran.txt", 'r'))
#titanTranConvert(open("g:\\Database\\TitanConvert\\8-Barnet\\barnettran.txt", 'r'))
def titanTranConvert(f):

    def startTran(margin):
        #margin = getMargins(ttext)

        fout.write('<?xml version="1.0"?>\n<TRN>\n<DocType>Summation Transcript Export File</DocType>\n<Version>0.9</Version>\n')
        fout.write('<Transcript_Name>' + title + '</Transcript_Name>\n')
        fout.write("<Transcript_Description>" + title +" - " + month +"/"+ day +"/"+ year +" - "+ location + "</Transcript_Description>\n")
        fout.write("<Notes>\n</Notes>\n")
        fout.write("<Transcript Info>\n-1,1," + str(margin) + ",0,0\n</Transcript Info>\n")
        fout.write('<Transcript>\n')

    def tranout(ttext):
            for line in ttext:
                fout.write(line+'\n')

    def getMargins(ttext):
        ml = [len(i[lineMargin:].rstrip()) - len(i[lineMargin:].strip()) for i in ttext if (len(i[lineMargin:].rstrip()) - len(i[lineMargin:].strip())) > 0]
        if len(ml) == 0:
            ml = [0]
        #return min(ml) + lineMargin
        return min(ml)

    #basepath = "g:\\Database\\TitanConvert\\1-Grace\\1-Privest\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\1-Grace\\2-Bentall\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\1-Grace\\3-RoyalMuseum\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\1-Grace\\4-Laurelton\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\1-Grace\\5-WRGrace\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\4-Amswiss\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\5-Alcan\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\6-Continental\\Transcripts"
    #basepath = "g:\\Database\\TitanConvert\\7-Fletcher\\Transcripts"
    basepath = "g:\\Database\\TitanConvert\\8-Barnet\\Transcripts"
    lineMargin = 5
    text = {}
    trans = {}
    month = ''
    day = ''
    year = ''
    location = ''
    title = ''
    transcriptPage = []
    pageno = ''
    margin = 0
    linenums = dict([])
    textid = 0
    firstTranscript = True
    summid = -1

    print("Loading file...")
    for trline in f:
        idsline = trline.split("\n")[0]
        if idsline.startswith("###"):
            if trans.has_key(summid):
                trans[summid][int(textid)] = [year, month, day, title, location, [pageno] + transcriptPage + [' ']]
            else:
                trans[summid]={int(textid): [year, month, day, title, location, [pageno] + transcriptPage + [' ']]}
            month = ''
            day = ''
            year = ''
            location = ''
            title = ''
            pageno = '00000'
            transcriptPage = []
            linenums = dict([])
           
        elif idsline.startswith("txt:"):
            #text
            ltxt = idsline.split(":", 1)[1]
            lineText = ltxt.split("=", 1)[1]
            lindex = ltxt.split("=", 1)[0]
            if linenums.has_key(lindex):
                liner = linenums[lindex].rjust(lineMargin) + lineText
            else:
                liner = ''.rjust(lineMargin) + lineText
            #if len(lineText.strip()) > 0:
            transcriptPage = transcriptPage + [liner]
        elif idsline.startswith("pageno:"):
            pageno = idsline.split("=", 1)[1].zfill(lineMargin)
        elif idsline.startswith("lineno:"):
            ln = idsline.split(":", 1)[1].split("=", 1)
            linenums[ln[0]] = ln[1]
        elif idsline.startswith("textid:1="):
            summid = idsline.split(":", 1)[1].split("=", 1)[1]
        elif idsline.startswith("docdate:1"): #month
            month = idsline.split("=", 1)[1]
        elif idsline.startswith("docdate:2"): #day
            day = idsline.split("=", 1)[1]
        elif idsline.startswith("docdate:3"): #year
            year = idsline.split("=", 1)[1]
        elif idsline.startswith("title:"):
            title = idsline.split("=", 1)[1]
        elif idsline.startswith("location:"):
            location = idsline.split("=", 1)[1]
        elif idsline.startswith("textid:2="):
            #tid = idsline.split(":", 1)[1].split("=", 1)
            #textid[tid[0]] = tid[1]
            textid = idsline.split(":", 1)[1].split("=", 1)[1]

    print("Outputting transcripts...")
    pagecount = 0
    print("Transcript count: ", len(trans.keys()))
    print(trans.keys())
    for sid in trans.keys(): #do each transcript
        fout = open(os.path.join(basepath, sid + '.TRN'), 'w') #open up transcript file for output
        pagelist = trans[sid].keys()    #get and sort page numbers
        pagelist.sort()
        print("Transcript: ",sid, "pages: ", len(pagelist))

        lines = []  #compile all lines for calculating margins
        for page in pagelist:
            curpage = trans[sid][page]
            lines = lines + curpage[5]
        marg = getMargins(lines)
        curpage = trans[sid][pagelist[0]]
        year = curpage[0]
        month = curpage[1]
        day = curpage[2]
        title = curpage[3]
        location = curpage[4]
        #print sid +" - "+ title
        startTran(marg)
        
        for page in pagelist:
            pagecount += 1
            curpage = trans[sid][page]

            year = curpage[0]
            month = curpage[1]
            day = curpage[2]
            title = curpage[3]
            location = curpage[4]
         
            tranout(curpage[5])
        fout.write("</Transcript>\n</TRN>\n")
        fout.close()
    print("Pages: ", pagecount)






def olddd():
    #for i in contents:
    for trline in f:
        i = trline.split("\n")[0]
        if i.startswith("###"):
            if (not firstTranscript) and summidPrev != summid:
                tranout(text, title, summid)
                text = {}
            month = pmonth
            day = pday
            year = pyear
            title = ptitle
            location = plocation
            #text = text + [pageno]
            #text = text + transcriptPage
            #text = text + [' ']
            text[int(pageno)] = [pageno] + transcriptPage + [' ']
            transcriptPage = []
            linenums = dict([])
            firstTranscript = False
        else:
            if i.startswith("txt:"):
                ltxt = i.split(":", 1)[1]
                lineText = ltxt.split("=", 1)[1]
                lindex = ltxt.split("=", 1)[0]
                liner = linenums[lindex].rjust(lineMargin) + lineText
                if len(lineText.strip()) > 0:
                    transcriptPage = transcriptPage + [liner]
            elif i.startswith("pageno:"):
                pageno = i.split("=", 1)[1].zfill(lineMargin)
            elif i.startswith("lineno:"):
                ln = i.split(":", 1)[1].split("=", 1)
                linenums[ln[0]] = ln[1]
            elif i.startswith("summid:"):
                summidPrev = summid
                summid = i.split(":", 1)[1].split("=", 1)[1]
            elif i.startswith("docdate:1"): #month
                pmonth = i.split("=", 1)[1]
            elif i.startswith("docdate:2"): #day
                pday = i.split("=", 1)[1]
            elif i.startswith("docdate:3"): #year
                pyear = i.split("=", 1)[1]
            elif i.startswith("title:"):
                ptitle = i.split("=", 1)[1]
                #for bann in '\\/:*?"<>|':
                #    ptitle = ptitle.replace(bann,"-")

            elif i.startswith("location:"):
                plocation = i.split("=", 1)[1]
    tranout(text,title, summid) #output last page

def summaryLineOut(line, delims=[',','"',';','~']):
    line = [delims[1] + i + delims[1] for i in line]
    return reduce(lambda x, y: x + delims[0] + y, line)
        

#titanSumConvert(open("z:\\TitanConvert\\Alcan\\alcanSumT.ids"))
#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\privdocs.ids"),open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\Loadfiles\\privdocs.txt", 'w'))
#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\privexh.ids"),open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\Loadfiles\\privexh.txt", 'w'))
#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\privnotes.ids"),open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\Loadfiles\\privnotes.txt", 'w'))
#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\privtstm.ids"),open("g:\\database\\TitanConvert\\1-Grace\\1-Privest\\Loadfiles\\privtstm.txt", 'w'))

#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\2-Bentall\\bentdocs.ids"),open("g:\\database\\TitanConvert\\1-Grace\\2-Bentall\\Loadfiles\\bentdocs.txt", 'w'))
#titanSumConvert(open("g:\\database\\TitanConvert\\1-Grace\\2-Bentall\\bentnote.ids"),open("g:\\database\\TitanConvert\\1-Grace\\2-Bentall\\Loadfiles\\bentnote.txt", 'w'))

#delims=[field delimiter, quote character, multi-entry delimiter, return value in data]
def titanSumConvert(f, outFile, delims=['|','^','>','~']):
    basepath = "z:\\TitanConvert\\Alcan"
    contents = f.readlines()
    contents = [i.split("\n")[0] for i in contents]
    summaries = []
    summaryRecord = dict([])
    fieldNames = []
    lines = 0
    records = 0
    datefields = ['date','docdate']
    multifields = []
    nodelim = []
    multlist = []
    for field in contents:
        if field.startswith("###"):
            #end of current summary
            summaries.append(summaryRecord)
            summaryRecord = dict([])
            records = records + 1
        else:
            fieldParts = idsField(field)
            if fieldParts[2]:
                if fieldNames.count(fieldParts[0]) == 0:
                    fieldNames = fieldNames + [fieldParts[0]]
                if summaryRecord.has_key(fieldParts[0]):
                    summaryRecord[fieldParts[0]] = summaryRecord[fieldParts[0]] + [fieldParts[2]]
                else:
                    summaryRecord[fieldParts[0]] = [fieldParts[2]]
        lines = lines + 1
    #print summaryLineOut(fieldNames, delims)
    outFile.write(summaryLineOut(fieldNames, delims) + '\n')
    for summary in summaries:
        line = []
        for fieldName in fieldNames:
            if datefields.count(fieldName) > 0:
                lineDelim = '/'
            elif multifields.count(fieldName) > 0:
                lineDelim = delims[2]
            elif nodelim.count(fieldName) > 0:
                lineDelim = ''
            else:
                lineDelim = delims[3]
            if summary.has_key(fieldName):
                fieldValue = summary[fieldName]
                if len(fieldValue) > 1:
                    if multlist.count(fieldName) == 0:
                        multlist.append(fieldName)
                for delim in delims:
                    for value in fieldValue:
                        if value.count(delim) > 0:
                            print("Delimiter "+ delim +" found in rownum "+ summary['rownum'][0] +" field "+ fieldName + " item", fieldValue.index(value), ":", fieldValue)
                line = line + [reduce(lambda x, y: x + lineDelim + y, fieldValue)]
                #print reduce(lambda x, y: x + lineDelim + y, summary[fieldName])  
            else:
                line = line + ['']
                #print ''
        outFile.write(summaryLineOut(line, delims) + '\n')
    print("Records:", records)
    print("Lines:", lines)
    print("Multiline fields:", multlist)
